#include "libdetag.h"
#include <libxml/HTMLparser.h>
#include <libxml/tree.h>
#include <tidy/tidy.h>
#include <tidy/buffio.h>
#include <iconv.h>
#include <fstream>
#include <set>
using namespace std;


static void print_element_names(xmlNode * a_node, set<string> &filters, vector<string> &tagset);
static void htmlTagFilter(xmlNodePtr node, set<string> &tagset);
static void getUrlInTag(xmlNodePtr node, vector<string> &urlset);
static void getUrlInContent(string content, vector<string> &urlset);
static void getNodeContent(xmlNodePtr root, string &content);
static void getTidiedHtml(const char* in, const int inLen, const char* encoding, string &out);
static xmlDocPtr htmlReadMemoryEx(const char *buffer, int size, const char *URL, const char *encoding, int options);
size_t code_convert(const char *from_encoding, const char *to_encoding,const char *from_str, const size_t from_str_len,char **to_str, size_t *to_str_len);
string ToUTF8(const string &source, string charset, int c = 0);
string ToUTF82(const string &source, string charset, int &r, int c = 0);

// tag set for used
static set<string> g_TagFilter;
static set<string> g_ContentFilter;
	
long initTagFilter(const char* cfgfile)
{
	getTagSet(cfgfile, g_TagFilter);
	return 0;
}

long initContentFilter(const char* cfgfile)
{
	getTagSet(cfgfile, g_ContentFilter);
	return 0;
}

long getDetachedTags(const char* file, const char* encoding, std::vector<std::string> &tagset)
{
	std::istream *is;
	is = new std::ifstream(file);
	if ( !*is )
	  return -1;
	
	int length;
	is->seekg(0, ios::end);
	length = is->tellg();
	is->seekg(0, ios::beg);

	char *buffer = new char[length+1];
	is->read(buffer, length);
	buffer[length] = 0;
	
	long ret = getDetachedTags(buffer, length, encoding, tagset);

	delete is;
	delete [] buffer;
	return ret;
}

long getDetachedTags(const char* inbuf, size_t inlen, const char* encoding, std::vector<std::string> &tagset)
{
  htmlDocPtr doc = NULL;
  htmlNodePtr root_element = NULL;
  htmlNodePtr cur_node = NULL;
  
  /*  
   * this initialize the library and check potential ABI mismatches
   * between the version it was compiled for and the actual shared
   * library used.
   */
  LIBXML_TEST_VERSION

  /*parse the file and get the DOM */
  doc = htmlReadMemoryEx(inbuf, inlen, NULL, encoding, HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
	
  if (doc == NULL) {
  	printf("error: could not parse html in memory.\n");
  }
  
  /*Get the root element node */
  root_element = xmlDocGetRootElement(doc);
	
  /*get detached tags*/
  print_element_names(root_element, g_TagFilter, tagset);
  
  /*free the document */
  xmlFreeDoc(doc);

  /*  
   *Free the global variables that may
   *have been allocated by the parser.
   */
  xmlCleanupParser();

  return tagset.size();
}

long getDetachedContent(const char* file, const char* encoding, std::string &content)
{
  std::istream *is;
  is = new std::ifstream(file);
  if ( !*is )
	return -1;

  int length;
  is->seekg(0, ios::end);
  length = is->tellg();
  is->seekg(0, ios::beg);
  char *buffer = new char[length+1];
  is->read(buffer, length);
  buffer[length] = 0;

  long ret = getDetachedContent(buffer, length, encoding, content);

  delete is;
  delete [] buffer;
  return ret;
}

long getDetachedContent(const char* inbuf, size_t inlen, const char* encoding, std::string &content)
{
  htmlDocPtr doc = NULL;
  htmlNodePtr root_element = NULL;
  
  /*  
   * this initialize the library and check potential ABI mismatches
   * between the version it was compiled for and the actual shared
   * library used.
   */
	LIBXML_TEST_VERSION

	/*parse the file and get the DOM */
	doc = htmlReadMemoryEx(inbuf, inlen, NULL, encoding, HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
	
	if (doc == NULL) {
  	printf("error: could not parse html in memory.\n");
  }
  
  /*Get the root element node */
  root_element = xmlDocGetRootElement(doc);

  /*delete ignore tags from html tree*/
  htmlTagFilter(root_element, g_ContentFilter);
	
  /*get content*/
  getNodeContent(root_element, content);
  
  /*free the document */
  xmlFreeDoc(doc);

  /*  
   *Free the global variables that may
   *have been allocated by the parser.
   */
  xmlCleanupParser();

	return 0;
}

long getUrlSet(const char* file, const char* encoding, std::vector<std::string> &urlset)
{
  std::istream *is;
  is = new std::ifstream(file);
  if ( !*is )
	return -1;

  int length;
  is->seekg(0, ios::end);
  length = is->tellg();
  is->seekg(0, ios::beg);
  char *buffer = new char[length+1];
  is->read(buffer, length);
  buffer[length] = 0;

  long ret = getUrlSet(buffer, length, encoding, urlset);

  delete is;
  delete [] buffer;
  return ret;
}

long getUrlSet(const char* inbuf, size_t inlen, const char* encoding, std::vector<std::string> &urlset)
{
  htmlDocPtr doc = NULL;
  htmlNodePtr root_element = NULL;
  htmlNodePtr cur_node = NULL;
  
  /*  
   * this initialize the library and check potential ABI mismatches
   * between the version it was compiled for and the actual shared
   * library used.
   */
  LIBXML_TEST_VERSION

  /*parse the file and get the DOM */
  doc = htmlReadMemoryEx(inbuf, inlen, NULL, encoding, HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
	
  if (doc == NULL) {
  	printf("error: could not parse html in memory.\n");
	return urlset.size();
  }
  
  /*Get the root element node */
  root_element = xmlDocGetRootElement(doc);
	
  /*get url in tag*/
  getUrlInTag(root_element, urlset);
  
  /*get url out tag*/
  string content;
  getDetachedContent(inbuf, inlen, encoding, content);
  getUrlInContent(content, urlset);
  
  /*free the document */
  xmlFreeDoc(doc);

  /*  
   *Free the global variables that may
   *have been allocated by the parser.
   */
  xmlCleanupParser();

  return urlset.size();
}


/****************************************************************************************************/


static void
print_element_names(xmlNode * a_node, std::set<std::string> &filters, std::vector<std::string> &tagset)
{
  xmlNode *cur_node = NULL;
  string tag;
  string property;
  string propvalue;
  string input;

  for (cur_node = a_node; cur_node; cur_node = cur_node->next) {
    if (cur_node->type == XML_ELEMENT_NODE) {
      tag = (char*)cur_node->name;
      
	  	if ( filters.count((char*)(cur_node->name)) == 1 ) {
		    xmlAttrPtr prop = cur_node->properties;
		    xmlChar *prop_value;
		    for ( ; prop != NULL ; prop = prop->next) {
		      if ( prop->name != NULL )
		      {
		        property = (char*)prop->name;
		        prop_value = xmlGetProp(cur_node, prop->name);
		        if ( prop_value != NULL )
		        {	
		        	input = tag;
		        	input += " ";
		        	input += property;
		        	input += "=";
		        	input += (char*)prop_value;
		          	tagset.push_back(input);
					xmlFree(prop_value);
		        }
		      }
		    }
		}
    }
    
    print_element_names(cur_node->children, filters, tagset);
  }
}

static void 
htmlTagFilter(xmlNodePtr node, set<string> &tagset)
{
	// traverse tree and delete node in tagset
	xmlNode *cur_node = node;
	while ( cur_node )
	{
		if ( cur_node->type == XML_ELEMENT_NODE && tagset.count(string((char*)cur_node->name)) )
		{// del node in tagsets
			xmlNodePtr tempNode;
			tempNode = cur_node->next;
			xmlUnlinkNode(cur_node);
      		xmlFreeNode(cur_node);
      		cur_node = tempNode;
      		continue;
		}

		htmlTagFilter(cur_node->children, tagset);
		cur_node = cur_node->next;
	}
}

static void getTagSet(const char* cfgfile, std::set<std::string> &tagset)
{
	// read filter tag to tagset.
    ifstream file(cfgfile);
    string line;
    if ( file != 0 ) 
    {   
	    while ( getline(file, line) )
        	tagset.insert(line);
    }   
}

static void getUrlInTag(xmlNodePtr a_node, std::vector<std::string> &urlset)
{
  xmlNode *cur_node = NULL;
  for (cur_node = a_node; cur_node; cur_node = cur_node->next) {
    if (cur_node->type == XML_ELEMENT_NODE) {
	    xmlAttrPtr prop = NULL;
	    for ( prop = cur_node->properties; prop != NULL ; prop = prop->next) {
        	char *prop_value = (char*)xmlGetProp(cur_node, prop->name);
        	if ( prop_value != NULL && strncmp(prop_value, "http://", strlen("http://")) == 0 ) {	
        		urlset.push_back(prop_value);
        	}
			if ( prop_value != NULL )
				xmlFree((xmlChar*)prop_value);
	    }
    }
    
    getUrlInTag(cur_node->children, urlset);
  }
}

static void getUrlInContent(string content, vector<string> &urlset)
{
	char *pCur = (char*)content.c_str();
	char *p = NULL;
	
	while ( (p = strstr(pCur, "http://")) != NULL )
	{
		int i = 0; 
		while ( strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_:/.?=&%", p[i]) != NULL )
		{
			i++;
		}
		
		urlset.push_back(string(p, i));
		pCur = p + i;
	}
}

static void getNodeContent(xmlNodePtr root, string &content)
{
	xmlNodePtr cur_node = NULL;
	for (cur_node = root; cur_node; cur_node = cur_node->next) {
		if ( cur_node->type == XML_TEXT_NODE ) {
			content += (char*) cur_node->content;
			content += " ";
		}

		getNodeContent(cur_node->children, content);
	}
}

static void getTidiedHtml(const char* in, const int inLen, const char* encoding, string &out)
{
	TidyBuffer output = {0};
	TidyBuffer errbuf = {0};
	int rc = -1; 
	Bool ok; 

	TidyDoc tdoc = tidyCreate();
	ok = tidyOptSetBool( tdoc, TidyXhtmlOut, yes );
	if ( ok )
	  	rc = tidySetCharEncoding(tdoc, encoding);
	if ( rc >= 0 )
		rc = tidySetErrorBuffer( tdoc, &errbuf );
	if ( rc >= 0 ) 
	    rc = tidyParseString( tdoc, in );
	if ( rc >= 0 ) 
	   	rc = tidyCleanAndRepair( tdoc ); 
	if ( rc >= 0 ) 
	    rc = tidyRunDiagnostics( tdoc );
	if ( rc > 1 )                           
	    rc = ( tidyOptSetBool(tdoc, TidyForceOutput, yes) ? rc : -1 );
	if ( rc >= 0 ) 
	    rc = tidySaveBuffer( tdoc, &output );
	if ( rc >= 0 )
	{
		out = (char*)output.bp;
	}
	else
	{
		out = in;
	}

	tidyBufFree( &output );
	tidyBufFree( &errbuf );
	tidyRelease( tdoc );
}

static xmlDocPtr htmlReadMemoryEx(const char *buffer, int size, const char *URL, const char *encoding, int options)
{
	//string tidied;
	//getTidiedHtml(buffer, size, "raw", tidied);
	//return htmlReadMemory(tidied.c_str(), tidied.length(), URL, encoding, options);

	xmlDocPtr pRet = NULL;

	if ( strcmp(encoding, "utf8") == 0 ||
		 strcmp(encoding, "utf-8") == 0 || 
		 strcmp(encoding, "UTF8") == 0 ||
		 strcmp(encoding, "UTF-8") == 0 )
	{
		pRet = htmlReadMemory(buffer, size, URL, "utf-8", options);
	}
	else
	{
		string source = buffer;
		string result;
		int rc;
		
		if ( strcmp(encoding, "gb2312") == 0 ||
			   strcmp(encoding, "gbk") == 0 )
		{
			result = ToUTF82(source, string("gb18030"), rc, 1);
		}
		else
		{
			result = ToUTF82(source, string(encoding), rc, 1);
		}

		if ( rc == 1 )
		{
			printf("libdetag: iconv error, maybe you input an error encoding.\n");
			return pRet;
		}

		pRet = htmlReadMemory(result.c_str(), result.length(), URL, "utf-8", options);
	}

	return pRet;
}

/*
static xmlDocPtr htmlReadMemoryEx(const char *buffer, int size, const char *URL, const char *encoding, int options)
{
	//string tidied;
	//getTidiedHtml(buffer, size, "raw", tidied);
	//return htmlReadMemory(tidied.c_str(), tidied.length(), URL, encoding, options);

	xmlDocPtr pRet = NULL;

	if ( strcmp(encoding, "utf8") == 0 ||
		 strcmp(encoding, "utf-8") == 0 || 
		 strcmp(encoding, "UTF8") == 0 ||
		 strcmp(encoding, "UTF-8") == 0 )
	{
		pRet = htmlReadMemory(buffer, size, URL, "utf-8", options);
	}
	else
	{
		char *dstBuf = NULL;
		size_t inlen = size;
		size_t outlen = 0; 
		size_t rc = 0;
		if ( strcmp(encoding, "gb2312") == 0 ||
			 strcmp(encoding, "gbk") == 0 )
			rc = code_convert("gb18030", "utf-8", buffer, inlen, &dstBuf, &outlen);
		else
			rc = code_convert((char*)encoding, "utf-8", buffer, inlen, &dstBuf, &outlen);

		if ( rc == -1 )
		{
			printf("libdetag: iconv error, maybe you input an error encoding.\n");
			return pRet;
		}

		pRet = htmlReadMemory(dstBuf, outlen, URL, "utf-8", options);
		if  ( dstBuf != NULL )
			free(dstBuf);
	}

	return pRet;
}
*/

/*
 *  Description:
 *    convert one type of encoding string to another type of encoding string using iconv
 *  Parameters:
 *    form_encoding:   type of source encoding
 *    to_encoding:     type of target encoding
 *    from_str:        encoding string of source type
 *    from_str_len:    length of encoding string of source type
 *    to_str[out]:     encoding string of target type, ending of string will be set to double zero "\00\00"
 *    to_str_len[out]: length of encoding string of target type
 *  Return:
 *    returns the number of characters converted in a non-reversible way during this call; reversible conversions are not counted. 
 *    In case of error, return -1
 *  WARNING:
 *    don't forget to call free() to release memory allocated for "to_str" pointer
 */
/*
size_t code_convert(const char *from_encoding, const char *to_encoding,const char *from_str, const size_t from_str_len,char **to_str, size_t *to_str_len)
{
	iconv_t icv;
	int     argument = 1;
	size_t  ret      = 0;
	size_t  n        = 1;

	const char *from = NULL;
	char *to         = NULL;
	char *p          = NULL;
	
	size_t from_len = 0;
	size_t to_len  = 0;

	 if((!from_encoding) || (!to_encoding) || (!from_str) || (from_str_len == 0)||(!to_str) || (!to_str_len))
		 return -1;

	icv = iconv_open(to_encoding,from_encoding);
	if( icv == (iconv_t)(-1) )
		return -1;

	//enable "illegal sequence discard and continue" feature, so that if met illeagal sequence, 
	//conversion will continue instead of being terminated
	if(iconvctl (icv ,ICONV_SET_DISCARD_ILSEQ,&argument) != 0)
		return -1;

	do
	{
		from = from_str;
		from_len = from_str_len;

		to_len = from_len * n;
		*to_str_len = to_len;
		to = (char*)malloc(to_len);
		if(!to)
		{
			iconv_close(icv);
			return -1;
		}
		*to_str = to;
		ret = iconv(icv,(char**)&from,&from_len,&to,&to_len);
		if(ret == -1)
		{
			// not enougt room of output buffer, we should reallocate more room for output buffer
			// just simply enlarge size of output buffer to x2 (x3 x4 and so on) size of input buffer
			if(errno == E2BIG)
			{
				n++;
				if ( *to_str != NULL )
				{
					free(*to_str);
					*to_str = NULL;
				}

				if ( n == 3 )
					return -1;
			}
			else
			{
				//something wrong, we should terminate conversation
				if( *to_str != NULL )
				{
					free(*to_str);
					*to_str = NULL;
				}
	
				iconv_close(icv);
				return -1;
			}
		}
	}while(ret == -1);

	iconv_close(icv);

	//size of output string
	*to_str_len -= to_len;

	//set the ending characters of output string to double zero "\00\00"
	p = *to_str;
	*to_str = (char*)malloc(*to_str_len + 2);
	memcpy(*to_str,p,*to_str_len);
	*(*to_str + *to_str_len) = 0;
	*(*to_str + *to_str_len + 1) = 0;
	free(p);

	return ret;
}
*/

string ToUTF8(const string &source, string charset, int c)
{
    string result;
	const char *from = charset.c_str();
	const char *to = "UTF-8";
	const char *src = source.c_str();
	size_t len = source.length();


	iconv_t cd;

	char *pinbuf = const_cast< char* >(src);
	size_t inbytesleft = len;
	char *poutbuf = NULL;
	size_t buf_size = 2 * len;
	size_t outbytesleft = buf_size;

	char *dst = NULL;
	size_t retbytes = 0;
	int done = 0;
	int errno_save = 0;

	if ((iconv_t)-1 == (cd = iconv_open(to, from))) 
	{
  		return source;
	}

	dst = new char[buf_size];

	while(inbytesleft > 0 && !done)  
	{
  		poutbuf = dst;
  		outbytesleft = buf_size;
  
  		retbytes = iconv(cd, &pinbuf, &inbytesleft, &poutbuf, &outbytesleft);
  		errno_save = errno;
  
  		if (dst != poutbuf)  {// we have something to write
   			result.append(dst, poutbuf-dst);
  		} 
  
  		if (retbytes != (size_t)-1) 
		{
   			poutbuf = dst;
   			outbytesleft = buf_size;
   			(void)iconv(cd, NULL, NULL, &poutbuf, &outbytesleft);
   
   			if (dst != poutbuf) 
			{// we have something to write
    			result.append(dst, poutbuf-dst);
   			}
   
   			errno_save = 0;
   			break;
  		}	
  
  
  
  		switch(errno_save)  {
  		case E2BIG:
   			break;
  		case EILSEQ:
   			if (c) {
    			errno_save = 0;
    			inbytesleft = len-(pinbuf-src); // forward one illegal byte
    			inbytesleft--;
				*pinbuf = ' ';
    			pinbuf++;
    			break;
   			}
   
   			done = 1;
   			break;
  		case EINVAL:
   			done = 1;
  	 		break;
  		default:
   			done = 1;
   			break; 
  		}
  
	}

	delete[] dst;
	iconv_close(cd);

	return result;
}

//r==0 success
//r==1 error
string ToUTF82(const string &source, string charset, int &r, int c)
{
	r = 0;

    string result;
	const char *from = charset.c_str();
	const char *to = "UTF-8";
	const char *src = source.c_str();
	size_t len = source.length();


	iconv_t cd;

	char *pinbuf = const_cast< char* >(src);
	size_t inbytesleft = len;
	char *poutbuf = NULL;
	size_t buf_size = 2 * len;
	size_t outbytesleft = buf_size;

	char *dst = NULL;
	size_t retbytes = 0;
	int done = 0;
	int errno_save = 0;

	if ((iconv_t)-1 == (cd = iconv_open(to, from))) 
	{
  		return source;
	}

	dst = new char[buf_size];

	while(inbytesleft > 0 && !done)  
	{
  		poutbuf = dst;
  		outbytesleft = buf_size;
  
  		retbytes = iconv(cd, &pinbuf, &inbytesleft, &poutbuf, &outbytesleft);
  		errno_save = errno;
  
  		if (dst != poutbuf)  {// we have something to write
   			result.append(dst, poutbuf-dst);
  		} 
  
  		if (retbytes != (size_t)-1) 
		{
   			poutbuf = dst;
   			outbytesleft = buf_size;
   			(void)iconv(cd, NULL, NULL, &poutbuf, &outbytesleft);
   
   			if (dst != poutbuf) 
			{// we have something to write
    			result.append(dst, poutbuf-dst);
   			}
   
   			errno_save = 0;
   			break;
  		}	
  
  
  
  		switch(errno_save)  {
  		case E2BIG:
   			break;
  		case EILSEQ:
			r = 1;
   			done = 1;
   			break;
  		case EINVAL:
   			done = 1;
  	 		break;
  		default:
   			done = 1;
   			break; 
  		}
  
	}

	delete[] dst;
	iconv_close(cd);

	return result;
}
